import json
import traceback

import pandas as pd

from air_web.data_platform import sql_engine

S_COLUMNS = [
    "terminal_addr",
    "coll_obj_id",
    "coll_port",
    "mp_id",
    "pt",
    "met_prot_code",
    "cis_tmnl_asset_no",
    "calc_mode",
    "erate_offset",
    "pq_flag",
    "encrypt_type",
    "freeze_cycle_num",
    "id",
    "install_date",
    "org_no",
    "create_date",
    "protocol_code",
    "wiring_mode",
    "cons_meas_mode",
    "baudrate",
    "met_volt_code",
    "fmr_asset_no",
    "area_code",
    "fc_gc_flag",
    "meter_digits",
    "mp_level",
    "asset_no",
    "area_no",
    "is_valid",
    "cons_no",
    "met_manufacturer",
    "cons_type",
    "mp_type",
    "mp_sn",
    "fee_ctrl_flag",
    "is_fqd",
    "comm_addr",
    "tmnl_asset_no",
    "cons_sort",
    "freeze_mode",
    "coll_mode",
    "factory_code",
    "data_src",
    "met_lot_no",
    "cons_name",
    "ct",
    "comm_no",
    "self_factor",
    "terminal_type_code",
    "tg_id",
    "t_factor",
    "met_meas_mode",
    "meter_id",
    "abso_mode",
    "self_ep_flag",
    "fmr_comm_addr",
    "gc_id",
]


def main(file):
    with open(file, "r") as f:
        data_list = []
        for idx, line in enumerate(f):
            data = json.loads(line)["data"]
            data_list.append(data)

            if idx % 1000 == 0:
                print(idx)
                df = pd.DataFrame(data_list)
                data_list = []

                # columns = df.columns.tolist()
                # if set(columns) != set(S_COLUMNS):
                #     in_columns = list(set(columns) & set(S_COLUMNS))
                #     df = df[in_columns]
                #     diff = set(columns) - set(S_COLUMNS)
                #     if len(diff) > 0:
                #         print('diff:', diff)

                try:
                    sql_engine.insert_df(df, "e_data_mp", drop_id=False)
                except:
                    print(traceback.format_exc())
        if data_list:
            df = pd.DataFrame(data_list)
            sql_engine.insert_df(df, "e_data_mp", drop_id=False)


if __name__ == "__main__":
    file = "/home/smxu/hebei/file/e.txt"
    main(file)
